/**
 * Module:		UTF/Unicode Conversations 
 * Authors:		Wei Li (oldrev@gmail.com)
 * License:		BSD
 * 
 * References:	http://www.cl.cam.ac.uk/~mgk25/unicode.html
 *				www.unicode.org
 */



module dotmars.text.utf;

import dotmars.base.string;
import dotmars.base.stdexcept;
import dotmars.base.stdtypes;
import dotmars.base.typetraits;


const ubyte[256] TrailingBytesForUTF8 = [
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
];

const dchar[6] OffsetsFromUTF8 = cast(dchar[6])[ 
	0x00000000, 0x00003080, 0x000E2080, 
	0x03C82080, 0xFA082080, 0x82082080 
];

const ubyte FirstByteMark[7] = [ 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC ];

////////////////////////////////////////////////////////////////////////////////

size_t stride(char c) {
	return TrailingBytesForUTF8[c] + 1;
}

size_t stride(wchar wc)
{
	size_t ret = 1;
	if(wc >= 0xd800 && wc <= 0xdbff)
		ret++;
    return ret; 
}

size_t stride(dchar dc) {
	return 1;
}


size_t count(string str)
in{
	assert(str !is null);
}
body{
	size_t n = 0;
	size_t i = 0;
	while(i < str.length){
		i += stride(str[i]);
		n ++;
	}
	return n;
}

size_t count(wstring wstr)
in{
	assert(wstr !is null);
}
body{
	size_t n = 0;
	size_t i = 0;
	while(i < wstr.length){
		i += stride(wstr[i]);
		n ++;
	}
	return n;
}

size_t count(dstring dstr)
in{
	assert(dstr !is null);
}
body{
	return dstr.length;
}

///////////////////////////////////////////////////////////////////////////////

void each(T)(T source, void delegate(size_t, dchar) proc)
in {
	assert(source !is null);
	assert(proc !is null);
}
body {

	size_t pos = 0;
	while(pos < source.length) {
		dchar d;
		size_t skip = toUtf32(source[pos..$], d);
		proc(pos, d);
		pos += skip;
	}
}


void each(T)(T source, void delegate(dchar) proc)
{
	each(source, 
			(size_t i, dchar d) { proc(d); }
	);
}


/////////////////////////////// Character-based encoding/decoding ////////////////////////////

size_t toUtf8(dchar d, Sink!(char) sink)
{
	dchar c = d;

	if (c <= 0x7f )
	{
		if(sink !is null) sink(c);
		return 1;
	}
	else if (c <= 0x7ff )
	{
		if(sink !is null){
			sink(0xc0 | ((c >> 6) & 0x3f));
			sink(0x80 | (c & 0x3f));
		}
		return 2;
	}
	else if (c <= 0xffff)
	{
		if(sink !is null){
			sink(0xe0 | ((c >> 12) & 0x3f));
			sink(0x80 | ((c >> 6)  & 0x3f));
			sink(0x80 | (c & 0x3f));
		}
		return 3;
	}
	else if (c <= 0x1fffff)
	{
		if(sink !is null){
			sink(0xf0 | ((c >> 18) & 0x3f));
			sink(0x80 | ((c >> 12) & 0x3f));
			sink(0x80 | ((c >> 6)  & 0x3f));
			sink(0x80 | (c & 0x3f));
		}
		return 4;
	}
	else //Only support to 4 bytes
		throw new ArgumentException ("toUtf8: Invalid character in dstr[])");
	return 0;
}

size_t toUtf16(dchar d, Sink!(wchar) sink)
{

	if(d <= 0xffff) 
	{
		if(sink !is null)
			sink(cast(wchar)d);
		return 1;
	}
	else {
		if(sink !is null){
	    	sink(cast(wchar) ((((d - 0x10000) >> 10) & 0x3FF) + 0xD800));
	    	sink(cast(wchar) (((d - 0x10000) & 0x3FF) + 0xDC00));
		}
		return 2;
	}
	
	return 0;
}


size_t toUtf32(string str, out dchar d)
in {
	assert(str.length > 0);
}
body {
	if(str.length == 0)
		throw new ArgumentException("Argument arg cannnot be empty");

	char* source = str.ptr;
	dchar ch = 0;

	size_t extraBytesToRead = TrailingBytesForUTF8[*source];
	
	if(str.length < extraBytesToRead + 1)
		throw new ArgumentException("Invalid UTF-8 sequence");

	//High! really is high! - oldrev
	switch (extraBytesToRead) 
	{
	    case 5: ch += *source++; ch <<= 6;
	    case 4: ch += *source++; ch <<= 6;
	    case 3: ch += *source++; ch <<= 6;
	    case 2: ch += *source++; ch <<= 6;
	    case 1: ch += *source++; ch <<= 6;
	    case 0: ch += *source++;
	}
	d = ch;
	return extraBytesToRead + 1;
}


size_t toUtf32(wstring wstr, out dchar d)
{
	size_t i = 0;
	dchar u = wstr[i];

	if (u & ~0x7F)
	{   
		if (u >= 0xD800 && u <= 0xDBFF)
		{   
			dchar u2;

			if (i + 1 == wstr.length)
			   throw new ArgumentException("surrogate UTF-16 high value past end of string");
			u2 = wstr[i + 1];
			if (u2 < 0xDC00 || u2 > 0xDFFF)
				throw new ArgumentException("surrogate UTF-16 low value out of range");
			u = ((u - 0xD7C0) << 10) + (u2 - 0xDC00);
			i += 2;
		}
		else if (u >= 0xDC00 && u <= 0xDFFF)
			throw new ArgumentException("unpaired surrogate UTF-16 value");
		else if (u == 0xFFFE || u == 0xFFFF)
		   throw new ArgumentException("Illegal UTF-16 value");
		else
			i++;
	}
	else
		i++;

	d = u;

	return i;
}

//Use UTF-32 as the intermediate result

// Inefficiency, but convenience.

string toUtf8(dchar d)
{
	string result;
	Sink!(char) sink = (char c) { result ~= c; };
	toUtf8(d, sink);
	return result;
}

wstring toUtf16(dchar d)
{
	wstring result;
	Sink!(wchar) sink = (wchar c) { result ~= c; };
	toUtf16(d, sink);
	return result;
}


string toUtf8(dstring dstr)
in{
	assert(dstr !is null);
}
body {
	string result;
	Sink!(char) sink = (char c) { result ~= c; };
	foreach(dchar d; dstr) {
		toUtf8(d, sink);
	}
	return result;
}

dstring toUtf32(string str)
in{
	assert(str !is null);
}
body {
	dstring result;

	Sink!(dchar) sink = (dchar c) { result ~= c; };

	size_t i = 0;
	while(i < str.length) {
		dchar d;
		size_t skip = toUtf32(str[i .. $], d);
		i += skip;
		sink(d);
	}

	return result;
}



dstring toUtf32(wstring wstr)
in{
	assert(wstr !is null);
}
body {
	dstring result;

	Sink!(dchar) sink = (dchar c) { result ~= c; };

	size_t i = 0;
	while(i < wstr.length) {
		dchar d;
		size_t skip = toUtf32(wstr[i .. $], d);
		i += skip;
		sink(d);
	}

	return result;
}


wstring toUtf16(dstring dstr)
in{
	assert(dstr !is null);
}
body{
	wstring result;

	Sink!(wchar) sink = (wchar c) { result ~= c; };

	foreach(dchar d; dstr) {
		toUtf16(d, sink);
	}

	return result;
}


string toUtf8(wstring wstr)
in{
	assert(wstr !is null);
}
body {
	string result;

	Sink!(char) sink = (char c) { result ~=  c; };

	size_t i = 0;
	while(i < wstr.length){
		dchar d;
		size_t skip = toUtf32(wstr[i..$], d);
		i += skip;
		toUtf8(d, sink);
	}

	return result;
}


wstring toUtf16(string str)
in{
	assert(str !is null);
}
body{
	wstring result;

	Sink!(wchar) sink = (wchar c) { result ~= c; };

	size_t i = 0;
	while(i < str.length) {
		dchar d;
		size_t skip = toUtf32(str[i .. $], d);
		i += skip;
		toUtf16(d, sink);
	}

	return result;
}


string toUtf8(wchar* s) {
    return toUtf8(toWstring(s));

}


wchar* toUtf16z(string s) {
    return toWstringz(toUtf16(s));
}

